{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib3\n",
    "from lxml import html\n",
    "from datetime import date\n",
    "import json\n",
    "\n",
    "http = urllib3.PoolManager()\n",
    "urllib3.disable_warnings()\n",
    "\n",
    "url_template = 'https://www.planecrashinfo.com/{0}/{0}-{1}.htm'\n",
    "max_year = date.today().year\n",
    "http_code = 200\n",
    "field_map = {\n",
    "    'Date:': 'date',\n",
    "    'Time:': 'time',\n",
    "    'Location:': 'location',\n",
    "    'Operator:': 'operator',\n",
    "    'Flight #:': 'flight_num',\n",
    "    'Route:': 'route',\n",
    "    'AC\\n        Type:': 'ac_type',\n",
    "    'Registration:': 'registration',\n",
    "    'cn / ln:': 'cn_ln',\n",
    "    'Aboard:': 'aboard',\n",
    "    'Fatalities:': 'fatalities',\n",
    "    'Ground:': 'ground',\n",
    "    'Summary:': 'summary'\n",
    "}\n",
    "data = []\n",
    "year = 1920\n",
    "while year <= max_year:\n",
    "    crash_id = 1\n",
    "    http_code = 200\n",
    "    while http_code == 200:\n",
    "        crash = {}\n",
    "        url = url_template.format(year, crash_id)\n",
    "        crash['url'] = url\n",
    "        crash['year'] = year\n",
    "        res = http.request(method='GET', url=url)\n",
    "        http_code = res.status\n",
    "        if http_code == 200:\n",
    "            print(f'parse {url}')\n",
    "            page = html.fromstring(res.data.decode('windows-1252'))\n",
    "            trs = page.xpath('//tr[position()>1]')\n",
    "            for tr in trs:\n",
    "                field = tr.xpath('./td[1]/font/b/text()')[0]\n",
    "                value = tr.xpath('./td[2]/font/text()')[0]\n",
    "                crash[field_map[field]] = value\n",
    "            crash_id += 1\n",
    "            data.append(crash)\n",
    "        else:\n",
    "            if http_code != 404:\n",
    "                print(f'fail {url}: {http_code}')\n",
    "    year += 1\n",
    "json.dump(obj=data, fp=open(f'plan-crash.json', mode='w', encoding='utf-8'), ensure_ascii=False)\n",
    "\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
